Deduplicate and fix up our use of mmap()
authorColin Walters <walters@verbum.org>
Wed, 4 Oct 2017 19:06:31 +0000 (15:06 -0400)
committerAtomic Bot <atomic-devel@projectatomic.io>
Wed, 4 Oct 2017 20:42:39 +0000 (20:42 +0000)
Buried in this large patch is a logical fix:

```
-  if (!map)
-    return glnx_throw_errno_prefix (error, "mmap");
+  if (map == (void*)-1)
+    return glnx_null_throw_errno_prefix (error, "mmap");
```

Which would have helped me debug another patch I was working
on.  But it turns out that actually correctly checking for
errors from `mmap()` triggers lots of other bugs - basically
because we sometimes handle zero-length variants (in detached
metadata).  When we start actually returning errors due to
this, things break.  (It wasn't a problem in practice before
because most things looked at the zero size, not the data).

Anyways there's a bigger picture issue here - a while ago
we made a fix to only use `mmap()` for reading metadata from disk
only if it was large enough (i.e. `>16k`).  But that didn't
help various other paths in the pull code and others that were
directly doing the `mmap()`.

Fix this by having a proper low level fs helper that does "read all data from
fd+offset into GBytes", which handles the size check. Then the `GVariant` bits
are just a clean layer on top of this. (At the small cost of an additional
allocation)

Side note: I had to remind myself, but the reason we can't just use
`GMappedFile` here is it doesn't support passing an offset into `mmap()`.

Closes: #1251
Approved by: jlebon

src/libostree/ostree-repo-commit.c
src/libostree/ostree-repo-pull.c
src/libostree/ostree-repo-static-delta-core.c
src/libostree/ostree-repo.c
src/libotutil/ot-fs-utils.c
src/libotutil/ot-fs-utils.h
src/libotutil/ot-variant-utils.c
src/libotutil/ot-variant-utils.h
src/ostree/ot-builtin-show.c
src/ostree/ot-builtin-summary.c

index 029dd01b5a9e93b3bae6734c52e392b093b2ce55..a9ae4af2b4d86170ccd0d271e638733948951e90 100644 (file)
@@ -2212,26 +2212,29 @@ ostree_repo_read_commit_detached_metadata (OstreeRepo      *self,
   char buf[_OSTREE_LOOSE_PATH_MAX];
   _ostree_loose_path (buf, checksum, OSTREE_OBJECT_TYPE_COMMIT_META, self->mode);
 
-  g_autoptr(GVariant) ret_metadata = NULL;
-  if (self->commit_stagedir.initialized &&
-      !ot_util_variant_map_at (self->commit_stagedir.fd, buf,
-                               G_VARIANT_TYPE ("a{sv}"),
-                               OT_VARIANT_MAP_ALLOW_NOENT | OT_VARIANT_MAP_TRUSTED, &ret_metadata, error))
-    return glnx_prefix_error (error, "Unable to read existing detached metadata");
-
-  if (ret_metadata == NULL &&
-      !ot_util_variant_map_at (self->objects_dir_fd, buf,
-                               G_VARIANT_TYPE ("a{sv}"),
-                               OT_VARIANT_MAP_ALLOW_NOENT | OT_VARIANT_MAP_TRUSTED, &ret_metadata, error))
-    return glnx_prefix_error (error, "Unable to read existing detached metadata");
-
-  if (ret_metadata == NULL && self->parent_repo)
+  if (self->commit_stagedir.initialized)
+    {
+      glnx_fd_close int fd = -1;
+      if (!ot_openat_ignore_enoent (self->commit_stagedir.fd, buf, &fd, error))
+        return FALSE;
+      if (fd != -1)
+        return ot_variant_read_fd (fd, 0, G_VARIANT_TYPE ("a{sv}"), TRUE,
+                                   out_metadata, error);
+    }
+
+  glnx_fd_close int fd = -1;
+  if (!ot_openat_ignore_enoent (self->objects_dir_fd, buf, &fd, error))
+    return FALSE;
+  if (fd != -1)
+    return ot_variant_read_fd (fd, 0, G_VARIANT_TYPE ("a{sv}"), TRUE,
+                               out_metadata, error);
+
+  if (self->parent_repo)
     return ostree_repo_read_commit_detached_metadata (self->parent_repo,
-                                                      checksum,
-                                                      out_metadata,
-                                                      cancellable,
-                                                      error);
-  ot_transfer_out_value (out_metadata, &ret_metadata);
+                                                      checksum, out_metadata,
+                                                      cancellable, error);
+  /* Nothing found */
+  *out_metadata = NULL;
   return TRUE;
 }
 
index a404b8a76c8d70e0967bcb7048cf1fdef058f41f..57bb19852f703da45e6420ea1cca22facf84a10a 100644 (file)
@@ -1228,8 +1228,8 @@ meta_fetch_on_complete (GObject           *object,
 
   if (fetch_data->is_detached_meta)
     {
-      if (!ot_util_variant_map_fd (fd, 0, G_VARIANT_TYPE ("a{sv}"),
-                                   FALSE, &metadata, error))
+      if (!ot_variant_read_fd (fd, 0, G_VARIANT_TYPE ("a{sv}"),
+                               FALSE, &metadata, error))
         goto out;
 
       if (!ostree_repo_write_commit_detached_metadata (pull_data->repo, checksum, metadata,
@@ -1245,8 +1245,8 @@ meta_fetch_on_complete (GObject           *object,
     }
   else
     {
-      if (!ot_util_variant_map_fd (fd, 0, ostree_metadata_variant_type (objtype),
-                                   FALSE, &metadata, error))
+      if (!ot_variant_read_fd (fd, 0, ostree_metadata_variant_type (objtype),
+                               FALSE, &metadata, error))
         goto out;
 
       /* Write the commitpartial file now while we're still fetching data */
index 85952b6a7ebf0a3c08a37a9fbe87f37af9ea544f..e5133a2bd23ee381e87aad8bfa5ee261eda066bd 100644 (file)
@@ -246,8 +246,8 @@ ostree_repo_static_delta_execute_offline (OstreeRepo                    *self,
     return glnx_throw_errno_prefix (error, "openat(%s)", basename);
 
   g_autoptr(GVariant) meta = NULL;
-  if (!ot_util_variant_map_fd (meta_fd, 0, G_VARIANT_TYPE (OSTREE_STATIC_DELTA_SUPERBLOCK_FORMAT),
-                               FALSE, &meta, error))
+  if (!ot_variant_read_fd (meta_fd, 0, G_VARIANT_TYPE (OSTREE_STATIC_DELTA_SUPERBLOCK_FORMAT),
+                           FALSE, &meta, error))
     return FALSE;
 
   /* Parsing OSTREE_STATIC_DELTA_SUPERBLOCK_FORMAT */
@@ -448,8 +448,8 @@ _ostree_static_delta_part_open (GInputStream   *part_in,
           int part_fd = g_file_descriptor_based_get_fd ((GFileDescriptorBased*)part_in);
 
           /* No compression, no checksums - a fast path */
-          if (!ot_util_variant_map_fd (part_fd, 1, G_VARIANT_TYPE (OSTREE_STATIC_DELTA_PART_PAYLOAD_FORMAT_V0),
-                                       trusted, &ret_part, error))
+          if (!ot_variant_read_fd (part_fd, 1, G_VARIANT_TYPE (OSTREE_STATIC_DELTA_PART_PAYLOAD_FORMAT_V0),
+                                   trusted, &ret_part, error))
             return FALSE;
         }
       else
@@ -767,9 +767,12 @@ _ostree_repo_static_delta_dump (OstreeRepo                    *self,
 
   superblock_path = _ostree_get_relative_static_delta_superblock_path (from, to);
 
-  if (!ot_util_variant_map_at (self->repo_dir_fd, superblock_path,
-                               (GVariantType*)OSTREE_STATIC_DELTA_SUPERBLOCK_FORMAT,
-                               OT_VARIANT_MAP_TRUSTED, &delta_superblock, error))
+  glnx_fd_close int superblock_fd = -1;
+  if (!glnx_openat_rdonly (self->repo_dir_fd, superblock_path, TRUE, &superblock_fd, error))
+    return FALSE;
+  if (!ot_variant_read_fd (superblock_fd, 0,
+                           (GVariantType*)OSTREE_STATIC_DELTA_SUPERBLOCK_FORMAT,
+                           TRUE, &delta_superblock, error))
     return FALSE;
 
   g_print ("Delta: %s\n", delta_id);
index 508291521c5a0979f6d799f52ef1996acfd160b9..4357d5622c340d0597b82c8649afdcdc1fbfd001 100644 (file)
@@ -2811,7 +2811,6 @@ load_metadata_internal (OstreeRepo       *self,
                         GError          **error)
 {
   char loose_path_buf[_OSTREE_LOOSE_PATH_MAX];
-  struct stat stbuf;
   glnx_fd_close int fd = -1;
   g_autoptr(GInputStream) ret_stream = NULL;
   g_autoptr(GVariant) ret_variant = NULL;
@@ -2853,36 +2852,14 @@ load_metadata_internal (OstreeRepo       *self,
 
   if (fd != -1)
     {
+      struct stat stbuf;
       if (!glnx_fstat (fd, &stbuf, error))
         return FALSE;
-
       if (out_variant)
         {
-          /* http://stackoverflow.com/questions/258091/when-should-i-use-mmap-for-file-access */
-          if (stbuf.st_size > 16*1024)
-            {
-              GMappedFile *mfile;
-
-              mfile = g_mapped_file_new_from_fd (fd, FALSE, error);
-              if (!mfile)
-                return FALSE;
-              ret_variant = g_variant_new_from_data (ostree_metadata_variant_type (objtype),
-                                                     g_mapped_file_get_contents (mfile),
-                                                     g_mapped_file_get_length (mfile),
-                                                     TRUE,
-                                                     (GDestroyNotify) g_mapped_file_unref,
-                                                     mfile);
-              g_variant_ref_sink (ret_variant);
-            }
-          else
-            {
-              g_autoptr(GBytes) data = glnx_fd_readall_bytes (fd, cancellable, error);
-              if (!data)
-                return FALSE;
-              ret_variant = g_variant_new_from_bytes (ostree_metadata_variant_type (objtype),
-                                                      data, TRUE);
-              g_variant_ref_sink (ret_variant);
-            }
+          if (!ot_variant_read_fd (fd, 0, ostree_metadata_variant_type (objtype), TRUE,
+                                   &ret_variant, error))
+            return FALSE;
 
           /* Now, let's put it in the cache */
           if (is_dirmeta_cachable)
@@ -4202,15 +4179,24 @@ ostree_repo_add_gpg_signature_summary (OstreeRepo     *self,
                                        GCancellable   *cancellable,
                                        GError        **error)
 {
-  g_autoptr(GBytes) summary_data = ot_file_mapat_bytes (self->repo_dir_fd, "summary", error);
+  glnx_fd_close int fd = -1;
+  if (!glnx_openat_rdonly (self->repo_dir_fd, "summary", TRUE, &fd, error))
+    return FALSE;
+  g_autoptr(GBytes) summary_data = ot_fd_readall_or_mmap (fd, 0, error);
   if (!summary_data)
     return FALSE;
+  /* Note that fd is reused below */
+  (void) close (glnx_steal_fd (&fd));
 
   g_autoptr(GVariant) existing_signatures = NULL;
-  if (!ot_util_variant_map_at (self->repo_dir_fd, "summary.sig",
-                               G_VARIANT_TYPE (OSTREE_SUMMARY_SIG_GVARIANT_STRING),
-                               OT_VARIANT_MAP_ALLOW_NOENT, &existing_signatures, error))
+  if (!ot_openat_ignore_enoent (self->repo_dir_fd, "summary.sig", &fd, error))
     return FALSE;
+  if (fd != -1)
+    {
+      if (!ot_variant_read_fd (fd, 0, G_VARIANT_TYPE (OSTREE_SUMMARY_SIG_GVARIANT_STRING),
+                               FALSE, &existing_signatures, error))
+        return FALSE;
+    }
 
   g_autoptr(GVariant) new_metadata = NULL;
   for (guint i = 0; key_id[i]; i++)
index 253de5b3ece71585f14a83d48804d786fb3d9192..79ba3cf7914026fc9b811aeb84b480a0017d6282 100644 (file)
@@ -22,6 +22,7 @@
 #include "ot-fs-utils.h"
 #include "libglnx.h"
 #include <sys/xattr.h>
+#include <sys/mman.h>
 #include <gio/gunixinputstream.h>
 #include <gio/gunixoutputstream.h>
 
@@ -144,22 +145,57 @@ ot_dfd_iter_init_allow_noent (int dfd,
   return TRUE;
 }
 
-GBytes *
-ot_file_mapat_bytes (int dfd,
-                     const char *path,
-                     GError **error)
-{
-  glnx_fd_close int fd = openat (dfd, path, O_RDONLY | O_CLOEXEC);
-  g_autoptr(GMappedFile) mfile = NULL;
+typedef struct {
+  gpointer addr;
+  gsize len;
+} MapData;
 
-  if (fd < 0)
-    return glnx_null_throw_errno_prefix (error, "openat(%s)", path);
+static void
+map_data_destroy (gpointer data)
+{
+  MapData *mdata = data;
+  (void) munmap (mdata->addr, mdata->len);
+  g_free (mdata);
+}
 
-  mfile = g_mapped_file_new_from_fd (fd, FALSE, error);
-  if (!mfile)
+/* Return a newly-allocated GBytes that refers to the contents of the file
+ * starting at offset @start. If the file is large enough, mmap() may be used.
+ */
+GBytes *
+ot_fd_readall_or_mmap (int           fd,
+                       goffset       start,
+                       GError      **error)
+{
+  struct stat stbuf;
+  if (!glnx_fstat (fd, &stbuf, error))
     return FALSE;
 
-  return g_mapped_file_get_bytes (mfile);
+  /* http://stackoverflow.com/questions/258091/when-should-i-use-mmap-for-file-access */
+  if (start > stbuf.st_size)
+    return g_bytes_new_static (NULL, 0);
+  const gsize len = stbuf.st_size - start;
+  if (len > 16*1024)
+    {
+      /* The reason we don't use g_mapped_file_new_from_fd() here
+       * is it doesn't support passing an offset, which is actually
+       * used by the static delta code.
+       */
+      gpointer map = mmap (NULL, len, PROT_READ, MAP_PRIVATE, fd, start);
+      if (map == (void*)-1)
+        return glnx_null_throw_errno_prefix (error, "mmap");
+
+      MapData *mdata = g_new (MapData, 1);
+      mdata->addr = map;
+      mdata->len = len;
+
+      return g_bytes_new_with_free_func (map, len, map_data_destroy, mdata);
+    }
+
+  /* Fall through to plain read into a malloc buffer */
+  if (lseek (fd, start, SEEK_SET) < 0)
+    return glnx_null_throw_errno_prefix (error, "lseek");
+  /* Not cancellable since this should be small */
+  return glnx_fd_readall_bytes (fd, NULL, error);
 }
 
 /* Given an input stream, splice it to an anonymous file (O_TMPFILE).
index 98fcd6a255d2edce38e74b5186077911a717c35f..7eb811cda1862578d7535397b83cc13ddc2d1870 100644 (file)
@@ -85,8 +85,7 @@ ot_map_anonymous_tmpfile_from_content (GInputStream *instream,
                                        GCancellable *cancellable,
                                        GError      **error);
 
-GBytes *ot_file_mapat_bytes (int dfd,
-                             const char *path,
-                             GError **error);
+GBytes *ot_fd_readall_or_mmap (int fd, goffset offset,
+                               GError **error);
 
 G_END_DECLS
index d1981c8bfffcf9670bd80ab0d81d979d4ed67864..d4ae7d9dad384a363230d7ea3b34690044c23545 100644 (file)
@@ -25,7 +25,6 @@
 #include <gio/gfiledescriptorbased.h>
 
 #include <string.h>
-#include <sys/mman.h>
 
 #include "otutil.h"
 
@@ -85,86 +84,25 @@ ot_util_variant_take_ref (GVariant *variant)
   return g_variant_take_ref (variant);
 }
 
+/* Create a GVariant in @out_variant that is backed by
+ * the data from @fd, starting at @start.  If the data is
+ * large enough, mmap() may be used.  @trusted is used
+ * by the GVariant core; see g_variant_new_from_data().
+ */
 gboolean
-ot_util_variant_map_at (int dfd,
-                        const char *path,
-                        const GVariantType *type,
-                        OtVariantMapFlags flags,
-                        GVariant **out_variant,
-                        GError  **error)
-{
-  glnx_fd_close int fd = -1;
-  const gboolean trusted = (flags & OT_VARIANT_MAP_TRUSTED) > 0;
-
-  fd = openat (dfd, path, O_RDONLY | O_CLOEXEC);
-  if (fd < 0)
-    {
-      if (errno == ENOENT && (flags & OT_VARIANT_MAP_ALLOW_NOENT) > 0)
-        {
-          *out_variant = NULL;
-          return TRUE;
-        }
-      else
-        {
-          glnx_set_error_from_errno (error);
-          g_prefix_error (error, "Opening %s: ", path);
-          return FALSE;
-        }
-    }
-
-  return ot_util_variant_map_fd (fd, 0, type, trusted, out_variant, error);
-}
-
-typedef struct {
-  gpointer addr;
-  gsize len;
-} VariantMapData;
-
-static void
-variant_map_data_destroy (gpointer data)
-{
-  VariantMapData *mdata = data;
-  (void) munmap (mdata->addr, mdata->len);
-  g_free (mdata);
-}
-
-gboolean
-ot_util_variant_map_fd (int                    fd,
-                        goffset                start,
-                        const GVariantType    *type,
-                        gboolean               trusted,
-                        GVariant             **out_variant,
-                        GError               **error)
+ot_variant_read_fd (int                    fd,
+                    goffset                start,
+                    const GVariantType    *type,
+                    gboolean               trusted,
+                    GVariant             **out_variant,
+                    GError               **error)
 {
-  gboolean ret = FALSE;
-  gpointer map;
-  struct stat stbuf;
-  VariantMapData *mdata = NULL;
-  gsize len;
-
-  if (fstat (fd, &stbuf) != 0)
-    {
-      glnx_set_error_from_errno (error);
-      goto out;
-    }
-
-  len = stbuf.st_size - start;
-  map = mmap (NULL, len, PROT_READ, MAP_PRIVATE, fd, start);
-  if (!map)
-    {
-      glnx_set_error_from_errno (error);
-      goto out;
-    }
-
-  mdata = g_new (VariantMapData, 1);
-  mdata->addr = map;
-  mdata->len = len;
+  g_autoptr(GBytes) bytes = ot_fd_readall_or_mmap (fd, start, error);
+  if (!bytes)
+    return FALSE;
 
-  ret = TRUE;
-  *out_variant = g_variant_ref_sink (g_variant_new_from_data (type, map, len, trusted,
-                                                              variant_map_data_destroy, mdata));
- out:
-  return ret;
+  *out_variant = g_variant_ref_sink (g_variant_new_from_bytes (type, bytes, trusted));
+  return TRUE;
 }
 
 GInputStream *
index 135ae5d01da74d96cac96e0d9a3f5d287c51f77a..379e19ac30604158810bec807d2b90ddcfb2e482 100644 (file)
@@ -36,24 +36,12 @@ GHashTable *ot_util_variant_asv_to_hash_table (GVariant *variant);
 
 GVariant * ot_util_variant_take_ref (GVariant *variant);
 
-typedef enum {
-  OT_VARIANT_MAP_TRUSTED = (1 << 0),
-  OT_VARIANT_MAP_ALLOW_NOENT = (1 << 1)
-} OtVariantMapFlags;
-
-gboolean ot_util_variant_map_at (int dfd,
-                                 const char *path,
-                                 const GVariantType *type,
-                                 OtVariantMapFlags flags,
-                                 GVariant **out_variant,
-                                 GError  **error);
-
-gboolean ot_util_variant_map_fd (int                  fd,
-                                 goffset              offset,
-                                 const GVariantType  *type,
-                                 gboolean             trusted,
-                                 GVariant           **out_variant,
-                                 GError             **error);
+gboolean ot_variant_read_fd (int                  fd,
+                             goffset              offset,
+                             const GVariantType  *type,
+                             gboolean             trusted,
+                             GVariant           **out_variant,
+                             GError             **error);
 
 GInputStream *ot_variant_read (GVariant             *variant);
 
index f197d7c381765758bcbd1e78e440f86bc813c029..3ab56eb2b8ba88206519b678bdce15391f7008ee 100644 (file)
@@ -58,7 +58,10 @@ do_print_variant_generic (const GVariantType *type,
 {
   g_autoptr(GVariant) variant = NULL;
 
-  if (!ot_util_variant_map_at (AT_FDCWD, filename, type, TRUE, &variant, error))
+  glnx_fd_close int fd = -1;
+  if (!glnx_openat_rdonly (AT_FDCWD, filename, TRUE, &fd, error))
+    return FALSE;
+  if (!ot_variant_read_fd (fd, 0, type, FALSE, &variant, error))
     return FALSE;
 
   ot_dump_variant (variant);
index abd1f86cc6899b6d9296ba8c8ff4f2082a7a81df..ed9e0b3d6c26f1ac35f60d7084bc72557eb622d9 100644 (file)
@@ -217,7 +217,10 @@ ostree_builtin_summary (int argc, char **argv, GCancellable *cancellable, GError
       if (opt_raw)
         flags |= OSTREE_DUMP_RAW;
 
-      summary_data = ot_file_mapat_bytes (repo->repo_dir_fd, "summary", error);
+      glnx_fd_close int fd = -1;
+      if (!glnx_openat_rdonly (repo->repo_dir_fd, "summary", TRUE, &fd, error))
+        return FALSE;
+      summary_data = ot_fd_readall_or_mmap (fd, 0, error);
       if (!summary_data)
         return FALSE;